{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4306e617-5118-4689-b6c6-80304c879ef6",
   "metadata": {},
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f05d69bb-0a43-4e8c-a192-69688c438fcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Importing plotly failed. Interactive plots will not work.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "from pyspark.sql import SparkSession\n",
    "from prophet import Prophet, serialize\n",
    "from prophet.diagnostics import cross_validation, performance_metrics\n",
    "import mlflow\n",
    "from mlflow.models import infer_signature\n",
    "\n",
    "DATASOURCE = (\n",
    "    \"/data/ts-spark_ch1_ds2.csv\"\n",
    ")\n",
    "ARTIFACT_DIR = \"model\"\n",
    "np.random.seed(20244)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "876cb937-c191-43c9-8441-028435bfe6d3",
   "metadata": {},
   "source": [
    "# DataOps"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fda2d5ff-5be6-4844-99e7-d60336904102",
   "metadata": {},
   "source": [
    "## Ingest data from source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "36951331-435e-468c-9a40-fd1cd3ec771d",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder \\\n",
    "    .master(\"spark://spark-master:7077\") \\\n",
    "    .appName(\"ts-spark_ch4_data-ml-ops_time_series_prophet_notebook\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "sdf = spark.read.csv(DATASOURCE, header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53f9b859-27ad-45b6-8668-45d11d14d1ef",
   "metadata": {},
   "source": [
    "## Transform data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0a87e661-31cd-4a12-b208-84a1b66376cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ds</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1981-01-01</td>\n",
       "      <td>20.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1981-01-02</td>\n",
       "      <td>17.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1981-01-03</td>\n",
       "      <td>18.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1981-01-04</td>\n",
       "      <td>14.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1981-01-05</td>\n",
       "      <td>15.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3643</th>\n",
       "      <td>1990-12-25</td>\n",
       "      <td>12.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3644</th>\n",
       "      <td>1990-12-26</td>\n",
       "      <td>14.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3645</th>\n",
       "      <td>1990-12-27</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3646</th>\n",
       "      <td>1990-12-28</td>\n",
       "      <td>13.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3647</th>\n",
       "      <td>1990-12-29</td>\n",
       "      <td>13.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3645 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              ds     y\n",
       "0     1981-01-01  20.7\n",
       "1     1981-01-02  17.9\n",
       "2     1981-01-03  18.8\n",
       "3     1981-01-04  14.6\n",
       "4     1981-01-05  15.8\n",
       "...          ...   ...\n",
       "3643  1990-12-25  12.9\n",
       "3644  1990-12-26  14.6\n",
       "3645  1990-12-27  14.0\n",
       "3646  1990-12-28  13.6\n",
       "3647  1990-12-29  13.5\n",
       "\n",
       "[3645 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pdf = sdf.select(\"date\", \"daily_min_temperature\").toPandas()\n",
    "pdf.columns = [\"ds\", \"y\"]\n",
    "pdf[\"y\"] = pd.to_numeric(pdf[\"y\"], errors=\"coerce\")\n",
    "pdf.drop(index=pdf.index[-2:], inplace=True)\n",
    "pdf.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c2c4305-21b4-424f-bd84-8d31d3b60be3",
   "metadata": {},
   "source": [
    "# ModelOps"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0918508-ec44-48cd-bd74-42a370aa8ead",
   "metadata": {},
   "source": [
    "## Train and log model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8af96d33-0c36-4748-96d9-3db618318879",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/04/30 18:49:50 INFO mlflow.tracking.fluent: Experiment with name 'ts-spark_ch4_data-ml-ops_time_series_prophet_notebook' does not exist. Creating a new experiment.\n",
      "18:49:51 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:51 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:53 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:53 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:53 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:53 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:54 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:57 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:58 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:58 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:58 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:58 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:58 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:49:59 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:59 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:59 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:49:59 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:00 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:01 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:01 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:01 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:01 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:02 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:03 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:04 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:04 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:04 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:04 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:04 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:04 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:04 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:05 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:06 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:06 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:07 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:07 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:08 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:08 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:09 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:09 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:09 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:09 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:10 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:11 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:11 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:12 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:12 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:12 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:12 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:13 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:13 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:14 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:14 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:15 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:15 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:15 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:15 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:15 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:16 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:16 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:16 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:17 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:17 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:17 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:18 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:18 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:18 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:18 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:18 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:19 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:20 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:20 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:20 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:20 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:20 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:20 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:21 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:21 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:21 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:22 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:22 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:22 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:22 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:23 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:23 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:23 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:24 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:24 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:24 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:25 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:25 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:26 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:26 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:27 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:27 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:27 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:27 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:27 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:28 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:28 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:28 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:28 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:29 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:29 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:30 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:30 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:31 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:31 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:32 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:32 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:33 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:33 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:33 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:34 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:34 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:35 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:35 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:35 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:36 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:36 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:36 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:36 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:36 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:37 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:37 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:37 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:38 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:38 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:38 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:39 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:39 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:40 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:41 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:41 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:41 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:41 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:41 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:42 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:42 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:43 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:43 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:44 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:44 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:44 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:44 - cmdstanpy - INFO - Chain [1] start processing\n",
      "18:50:44 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:45 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:45 - cmdstanpy - INFO - Chain [1] done processing\n",
      "18:50:45 - cmdstanpy - INFO - Chain [1] done processing\n",
      "/opt/conda/lib/python3.11/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\n",
      "  warnings.warn(\"Setuptools is replacing distutils.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV params: \n",
      "{\n",
      "  \"growth\": \"linear\",\n",
      "  \"n_changepoints\": 25,\n",
      "  \"specified_changepoints\": false,\n",
      "  \"changepoint_range\": 0.8,\n",
      "  \"yearly_seasonality\": \"auto\",\n",
      "  \"weekly_seasonality\": \"auto\",\n",
      "  \"daily_seasonality\": \"auto\",\n",
      "  \"seasonality_mode\": \"additive\",\n",
      "  \"seasonality_prior_scale\": 10.0,\n",
      "  \"changepoint_prior_scale\": 0.05,\n",
      "  \"holidays_prior_scale\": 10.0,\n",
      "  \"mcmc_samples\": 0,\n",
      "  \"interval_width\": 0.8,\n",
      "  \"uncertainty_samples\": 1000,\n",
      "  \"y_scale\": 26.3,\n",
      "  \"y_min\": 0.0,\n",
      "  \"scaling\": \"absmax\",\n",
      "  \"logistic_floor\": false,\n",
      "  \"country_holidays\": null,\n",
      "  \"component_modes\": {\n",
      "    \"additive\": [\n",
      "      \"yearly\",\n",
      "      \"weekly\",\n",
      "      \"additive_terms\",\n",
      "      \"extra_regressors_additive\",\n",
      "      \"holidays\"\n",
      "    ],\n",
      "    \"multiplicative\": [\n",
      "      \"multiplicative_terms\",\n",
      "      \"extra_regressors_multiplicative\"\n",
      "    ]\n",
      "  },\n",
      "  \"holidays_mode\": \"additive\"\n",
      "}\n",
      "CV metrics: \n",
      "{\n",
      "  \"mse\": 7.933370571149654,\n",
      "  \"rmse\": 2.816072480474402,\n",
      "  \"mae\": 2.2336426897094794,\n",
      "  \"mdape\": 0.1743597502078967,\n",
      "  \"smape\": 0.22119437920557894,\n",
      "  \"coverage\": 0.8004637054988742\n",
      "}\n",
      "Model URI: mlflow-artifacts:/2/e72065f3bf574795886f5b1be6a58059/artifacts/model\n"
     ]
    }
   ],
   "source": [
    "mlflow.set_tracking_uri(\"http://mlflow-server:5000\")\n",
    "mlflow.set_experiment('ts-spark_ch4_data-ml-ops_time_series_prophet_notebook')\n",
    "\n",
    "with mlflow.start_run():    \n",
    "    model = Prophet().fit(pdf)\n",
    "\n",
    "    param = {attr: getattr(model, attr) for attr in serialize.SIMPLE_ATTRIBUTES}\n",
    "\n",
    "    cv_metrics_name = [\"mse\", \"rmse\", \"mae\", \"mdape\", \"smape\", \"coverage\"]\n",
    "    cv_params = cross_validation(\n",
    "        model=model,\n",
    "        horizon=\"90 days\",\n",
    "        period=\"30 days\",\n",
    "        initial=\"700 days\",\n",
    "        parallel=\"threads\",\n",
    "        disable_tqdm=True,\n",
    "    )\n",
    "    _cv_metrics = performance_metrics(cv_params)\n",
    "    cv_metrics = {n: _cv_metrics[n].mean() for n in cv_metrics_name}\n",
    "\n",
    "    train = model.history\n",
    "    predictions = model.predict(model.make_future_dataframe(30))\n",
    "    signature = infer_signature(train, predictions)\n",
    "\n",
    "    mlflow.prophet.log_model(model, artifact_path=ARTIFACT_DIR, signature=signature)\n",
    "    mlflow.log_params(param)\n",
    "    mlflow.log_metrics(cv_metrics)\n",
    "    model_uri = mlflow.get_artifact_uri(ARTIFACT_DIR)\n",
    "\n",
    "    print(f\"CV params: \\n{json.dumps(param, indent=2)}\")\n",
    "    print(f\"CV metrics: \\n{json.dumps(cv_metrics, indent=2)}\")\n",
    "    print(f\"Model URI: {model_uri}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0754ef2-be34-4bb2-aa95-40f0269f4b71",
   "metadata": {},
   "source": [
    "## Forecast with model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "47d118d7-5e9b-4ccd-88c5-535de0e754e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03556aebd66c406189713e1530ce6c13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading artifacts:   0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "forecast:\n",
      "$          ds      trend  yhat_lower  yhat_upper  trend_lower  trend_upper  \\\n",
      "0 1981-01-01  11.839497   11.874776   18.524958    11.839497    11.839497   \n",
      "1 1981-01-02  11.837740   11.601939   18.410193    11.837740    11.837740   \n",
      "2 1981-01-03  11.835982   12.056064   18.841682    11.835982    11.835982   \n",
      "3 1981-01-04  11.834224   11.864106   18.528551    11.834224    11.834224   \n",
      "4 1981-01-05  11.832467   12.165678   18.753201    11.832467    11.832467   \n",
      "5 1981-01-06  11.830709   12.371939   19.258229    11.830709    11.830709   \n",
      "6 1981-01-07  11.828951   12.106755   19.083532    11.828951    11.828951   \n",
      "7 1981-01-08  11.827194   12.319995   19.202247    11.827194    11.827194   \n",
      "8 1981-01-09  11.825436   12.524928   19.314604    11.825436    11.825436   \n",
      "9 1981-01-10  11.823678   12.231800   19.161781    11.823678    11.823678   \n",
      "\n",
      "   additive_terms  additive_terms_lower  additive_terms_upper    weekly  \\\n",
      "0        3.317405              3.317405              3.317405  0.005744   \n",
      "1        3.364660              3.364660              3.364660 -0.025061   \n",
      "2        3.414020              3.414020              3.414020 -0.053369   \n",
      "3        3.382855              3.382855              3.382855 -0.160550   \n",
      "4        3.594918              3.594918              3.594918 -0.021611   \n",
      "5        3.771064              3.771064              3.771064  0.085487   \n",
      "6        3.918817              3.918817              3.918817  0.169360   \n",
      "7        3.812938              3.812938              3.812938  0.005744   \n",
      "8        3.832904              3.832904              3.832904 -0.025061   \n",
      "9        3.847750              3.847750              3.847750 -0.053369   \n",
      "\n",
      "   weekly_lower  weekly_upper    yearly  yearly_lower  yearly_upper  \\\n",
      "0      0.005744      0.005744  3.311660      3.311660      3.311660   \n",
      "1     -0.025061     -0.025061  3.389721      3.389721      3.389721   \n",
      "2     -0.053369     -0.053369  3.467389      3.467389      3.467389   \n",
      "3     -0.160550     -0.160550  3.543405      3.543405      3.543405   \n",
      "4     -0.021611     -0.021611  3.616529      3.616529      3.616529   \n",
      "5      0.085487      0.085487  3.685577      3.685577      3.685577   \n",
      "6      0.169360      0.169360  3.749457      3.749457      3.749457   \n",
      "7      0.005744      0.005744  3.807194      3.807194      3.807194   \n",
      "8     -0.025061     -0.025061  3.857965      3.857965      3.857965   \n",
      "9     -0.053369     -0.053369  3.901119      3.901119      3.901119   \n",
      "\n",
      "   multiplicative_terms  multiplicative_terms_lower  \\\n",
      "0                   0.0                         0.0   \n",
      "1                   0.0                         0.0   \n",
      "2                   0.0                         0.0   \n",
      "3                   0.0                         0.0   \n",
      "4                   0.0                         0.0   \n",
      "5                   0.0                         0.0   \n",
      "6                   0.0                         0.0   \n",
      "7                   0.0                         0.0   \n",
      "8                   0.0                         0.0   \n",
      "9                   0.0                         0.0   \n",
      "\n",
      "   multiplicative_terms_upper       yhat  \n",
      "0                         0.0  15.156902  \n",
      "1                         0.0  15.202400  \n",
      "2                         0.0  15.250002  \n",
      "3                         0.0  15.217079  \n",
      "4                         0.0  15.427384  \n",
      "5                         0.0  15.601773  \n",
      "6                         0.0  15.747768  \n",
      "7                         0.0  15.640131  \n",
      "8                         0.0  15.658340  \n",
      "9                         0.0  15.671428  \n"
     ]
    }
   ],
   "source": [
    "_model = mlflow.prophet.load_model(model_uri)\n",
    "\n",
    "forecast = _model.predict(_model.make_future_dataframe(30))\n",
    "print(f\"forecast:\\n${forecast.head(10)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
